{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "640c9e97",
   "metadata": {
    "papermill": {
     "duration": 0.0061,
     "end_time": "2023-04-08T14:38:47.617606",
     "exception": false,
     "start_time": "2023-04-08T14:38:47.611506",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Conv1D TF Model with Separate Pipelines for defog and tdcsfog data\n",
    "Feature Column TimeSeries grouping adapted from https://www.kaggle.com/code/mayukh18/pytorch-fog-end-to-end-baseline-lb-0-254\n",
    "\n",
    "Subject-wise GroupKFold splitting adapted from https://www.kaggle.com/code/xzj19013742/groupkfold-cross-validation-tsflex\n",
    "### In this Notebook\n",
    "- Tensorflow Model with Conv1D blocks \n",
    "- Models trained separately for defog and tdcsfog data\n",
    "- Event Stratified Subject Grouped KFold splitting"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e6836da9",
   "metadata": {
    "papermill": {
     "duration": 0.004512,
     "end_time": "2023-04-08T14:38:47.627046",
     "exception": false,
     "start_time": "2023-04-08T14:38:47.622534",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Imports and Config"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "3929dddb",
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5",
    "execution": {
     "iopub.execute_input": "2023-04-08T14:38:47.639264Z",
     "iopub.status.busy": "2023-04-08T14:38:47.638021Z",
     "iopub.status.idle": "2023-04-08T14:38:55.186535Z",
     "shell.execute_reply": "2023-04-08T14:38:55.184994Z"
    },
    "papermill": {
     "duration": 7.557192,
     "end_time": "2023-04-08T14:38:55.188993",
     "exception": false,
     "start_time": "2023-04-08T14:38:47.631801",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TF version: 2.11.0\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import gc\n",
    "import numpy as np\n",
    "from numpy.random import default_rng\n",
    "import pandas as pd\n",
    "from tqdm.auto import tqdm\n",
    "from glob import glob\n",
    "from os.path import basename, dirname, join, exists\n",
    "from time import perf_counter\n",
    "from collections import defaultdict as dd\n",
    "from functools import partial\n",
    "\n",
    "from sklearn.model_selection import train_test_split, StratifiedKFold, StratifiedGroupKFold\n",
    "from sklearn.metrics import average_precision_score\n",
    "from sklearn.preprocessing import StandardScaler as Scaler\n",
    "from scipy.special import expit\n",
    "\n",
    "import tensorflow as tf\n",
    "print(f\"TF version: {tf.__version__}\")\n",
    "AUTO = tf.data.experimental.AUTOTUNE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "d051cf3e",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-04-08T14:38:55.201219Z",
     "iopub.status.busy": "2023-04-08T14:38:55.200405Z",
     "iopub.status.idle": "2023-04-08T14:38:55.226324Z",
     "shell.execute_reply": "2023-04-08T14:38:55.225433Z"
    },
    "papermill": {
     "duration": 0.033956,
     "end_time": "2023-04-08T14:38:55.228212",
     "exception": false,
     "start_time": "2023-04-08T14:38:55.194256",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Constants\n",
    "\n",
    "BASE_DIR = \"/kaggle/input/tlvmc-parkinsons-freezing-gait-prediction\"\n",
    "TRAIN_DIR = join(BASE_DIR, \"train\")\n",
    "TEST_DIR = join(BASE_DIR, \"test\")\n",
    "\n",
    "IS_PUBLIC = len(glob(join(TEST_DIR, \"*/*.csv\")))==2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "086b9a4f",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-04-08T14:38:55.239629Z",
     "iopub.status.busy": "2023-04-08T14:38:55.238805Z",
     "iopub.status.idle": "2023-04-08T14:38:55.246523Z",
     "shell.execute_reply": "2023-04-08T14:38:55.245647Z"
    },
    "papermill": {
     "duration": 0.0155,
     "end_time": "2023-04-08T14:38:55.248592",
     "exception": false,
     "start_time": "2023-04-08T14:38:55.233092",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "class Config:\n",
    "    train_sub_dirs = [\n",
    "        join(TRAIN_DIR, \"defog\"),\n",
    "        join(TRAIN_DIR, \"tdcsfog\")\n",
    "    ]\n",
    "    \n",
    "    metadata_paths = [\n",
    "        join(BASE_DIR, \"defog_metadata.csv\"),\n",
    "        join(BASE_DIR, \"tdcsfog_metadata.csv\")\n",
    "    ]\n",
    "    \n",
    "    splits = 10\n",
    "\n",
    "    batch_size = 1024\n",
    "    window_size = 64\n",
    "    window_future = 16\n",
    "    window_past = window_size - window_future # Includes current value\n",
    "    \n",
    "    wx = 8\n",
    "    \n",
    "    model_dropout = 0.2\n",
    "    model_hidden = 128\n",
    "    model_nblocks = 3\n",
    "    \n",
    "    lr = 0.00015\n",
    "    num_epochs = 5\n",
    "    \n",
    "    feature_list = ['AccV', 'AccML', 'AccAP']\n",
    "    label_list = ['StartHesitation', 'Turn', 'Walking']\n",
    "    \n",
    "    n_features = len(feature_list)\n",
    "    n_labels = len(label_list)    \n",
    "    \n",
    "cfg = Config()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6a0a8790",
   "metadata": {
    "papermill": {
     "duration": 0.004601,
     "end_time": "2023-04-08T14:38:55.259046",
     "exception": false,
     "start_time": "2023-04-08T14:38:55.254445",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Stratified Group K Fold"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "304642f9",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-04-08T14:38:55.270494Z",
     "iopub.status.busy": "2023-04-08T14:38:55.269698Z",
     "iopub.status.idle": "2023-04-08T14:38:55.309532Z",
     "shell.execute_reply": "2023-04-08T14:38:55.308441Z"
    },
    "papermill": {
     "duration": 0.048218,
     "end_time": "2023-04-08T14:38:55.312072",
     "exception": false,
     "start_time": "2023-04-08T14:38:55.263854",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "id2sub_df length: 970, unique Ids: 970, unique Subjects: 107\n"
     ]
    }
   ],
   "source": [
    "# Create Mapping between Id and Subject\n",
    "id2sub_df = pd.concat([\n",
    "    pd.read_csv(f, usecols=['Id', 'Subject']).assign(Module=basename(f).split('_')[0]) for f in cfg.metadata_paths\n",
    "]).astype(\"category\").set_index(\"Id\")\n",
    "print(f\"id2sub_df length: {len(id2sub_df)}, unique Ids: {id2sub_df.index.nunique()}, unique Subjects: {id2sub_df.Subject.nunique()}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "01f2724a",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-04-08T14:38:55.323658Z",
     "iopub.status.busy": "2023-04-08T14:38:55.322850Z",
     "iopub.status.idle": "2023-04-08T14:38:55.331078Z",
     "shell.execute_reply": "2023-04-08T14:38:55.329947Z"
    },
    "papermill": {
     "duration": 0.016298,
     "end_time": "2023-04-08T14:38:55.333278",
     "exception": false,
     "start_time": "2023-04-08T14:38:55.316980",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Read csv files and add metadata (Id, Subject, Event)\n",
    "def reader(filepath, usecols, getid=False, getsub=False, getevent=False, dtype=None, exclude=['notype']):\n",
    "    fog_type = basename(dirname(filepath))\n",
    "    if fog_type in exclude:\n",
    "        return None\n",
    "    df = pd.read_csv(filepath, index_col=\"Time\", usecols=usecols, dtype=dtype)\n",
    "    if getid:\n",
    "        df['Id'] = basename(filepath).split('.')[0] + '_' + df.index.astype(str)\n",
    "    if getsub:\n",
    "        df['Subject'] = id2sub_df.loc[basename(filepath).split('.')[0], 'Subject']\n",
    "    if getevent:\n",
    "        df['Event'] = np.select(\n",
    "            [df[col].astype(bool) for col in cfg.label_list], \n",
    "            np.arange(1,cfg.n_labels+1), default=0\n",
    "        ).astype('int8')\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "71a66579",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-04-08T14:38:55.344508Z",
     "iopub.status.busy": "2023-04-08T14:38:55.343835Z",
     "iopub.status.idle": "2023-04-08T14:39:32.388085Z",
     "shell.execute_reply": "2023-04-08T14:39:32.387158Z"
    },
    "papermill": {
     "duration": 37.052145,
     "end_time": "2023-04-08T14:39:32.390267",
     "exception": false,
     "start_time": "2023-04-08T14:38:55.338122",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2bb68e4192c24d8dac58e420f90fd18b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/970 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<style type=\"text/css\">\n",
       "#T_31c04_row0_col0 {\n",
       "  background-color: #023858;\n",
       "  color: #f1f1f1;\n",
       "}\n",
       "#T_31c04_row1_col0 {\n",
       "  background-color: #eee9f3;\n",
       "  color: #000000;\n",
       "}\n",
       "#T_31c04_row2_col0, #T_31c04_row3_col0 {\n",
       "  background-color: #fff7fb;\n",
       "  color: #000000;\n",
       "}\n",
       "</style>\n",
       "<table id=\"T_31c04_\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th class=\"blank level0\" >&nbsp;</th>\n",
       "      <th class=\"col_heading level0 col0\" >Event</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th id=\"T_31c04_level0_row0\" class=\"row_heading level0 row0\" >0</th>\n",
       "      <td id=\"T_31c04_row0_col0\" class=\"data row0 col0\" >17735798</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_31c04_level0_row1\" class=\"row_heading level0 row1\" >2</th>\n",
       "      <td id=\"T_31c04_row1_col0\" class=\"data row1 col0\" >2247047</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_31c04_level0_row2\" class=\"row_heading level0 row2\" >1</th>\n",
       "      <td id=\"T_31c04_row2_col0\" class=\"data row2 col0\" >305290</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_31c04_level0_row3\" class=\"row_heading level0 row3\" >3</th>\n",
       "      <td id=\"T_31c04_row3_col0\" class=\"data row3 col0\" >300239</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n"
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x74fa5d735350>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Create common train Dataframe\n",
    "train_paths = glob(join(TRAIN_DIR, '*/*.csv'))\n",
    "dtype = {col:'int8' for col in cfg.label_list}\n",
    "dtype['Time'] = 'int32'\n",
    "usecols = ['Time', *cfg.label_list]\n",
    "\n",
    "train_reader = partial(reader, usecols=usecols, dtype=dtype, getsub=True, getevent=True)\n",
    "train_df = pd.concat([train_reader(f) for f in tqdm(train_paths)]).reset_index(drop=True)\n",
    "train_df.Subject = train_df.Subject.astype('category')\n",
    "display(train_df.Event.value_counts().to_frame().style.background_gradient())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "69eef79c",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-04-08T14:39:32.403214Z",
     "iopub.status.busy": "2023-04-08T14:39:32.401722Z",
     "iopub.status.idle": "2023-04-08T14:40:45.749020Z",
     "shell.execute_reply": "2023-04-08T14:40:45.747772Z"
    },
    "papermill": {
     "duration": 73.356706,
     "end_time": "2023-04-08T14:40:45.752266",
     "exception": false,
     "start_time": "2023-04-08T14:39:32.395560",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "defog:\n",
      "\tFold 0: Subjects->train:35|test:3\n",
      "\tFold 1: Subjects->train:34|test:4\n",
      "\tFold 2: Subjects->train:35|test:3\n",
      "\tFold 3: Subjects->train:34|test:4\n",
      "\tFold 4: Subjects->train:35|test:3\n",
      "\tFold 5: Subjects->train:33|test:5\n",
      "\tFold 6: Subjects->train:34|test:4\n",
      "\tFold 7: Subjects->train:34|test:4\n",
      "\tFold 8: Subjects->train:34|test:4\n",
      "\tFold 9: Subjects->train:34|test:4\n",
      "tdcsfog:\n",
      "\tFold 0: Subjects->train:55|test:7\n",
      "\tFold 1: Subjects->train:56|test:6\n",
      "\tFold 2: Subjects->train:56|test:6\n",
      "\tFold 3: Subjects->train:56|test:6\n",
      "\tFold 4: Subjects->train:56|test:6\n",
      "\tFold 5: Subjects->train:55|test:7\n",
      "\tFold 6: Subjects->train:56|test:6\n",
      "\tFold 7: Subjects->train:56|test:6\n",
      "\tFold 8: Subjects->train:56|test:6\n",
      "\tFold 9: Subjects->train:56|test:6\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Save paths for each Stratified Group Fold for defog and tdcsfog separately\n",
    "sgkf = StratifiedGroupKFold(n_splits=cfg.splits, random_state=42, shuffle=True)\n",
    "fold_train_fpaths, fold_valid_fpaths = {'defog': [], 'tdcsfog':[]}, {'defog': [], 'tdcsfog':[]}\n",
    "df_paths = {'defog':glob(join(cfg.train_sub_dirs[0],'*.csv')), 'tdcsfog': glob(join(cfg.train_sub_dirs[1],'*.csv'))}\n",
    "for module, paths in df_paths.items():\n",
    "    print(f\"{module}:\")\n",
    "    sub_train_df = train_df[train_df.Subject.isin(id2sub_df.loc[id2sub_df.Module==module, 'Subject'])].reset_index(drop=True)\n",
    "    for i, (train_index, test_index) in enumerate(sgkf.split(sub_train_df.index, sub_train_df.Event, groups=sub_train_df.Subject)):\n",
    "        print(f\"\\tFold {i}:\", end=\" \")\n",
    "        train_subs = sub_train_df.loc[train_index, 'Subject'].unique()\n",
    "        test_subs = sub_train_df.loc[test_index, 'Subject'].unique()\n",
    "        print(f\"Subjects->train:{len(train_subs)}|test:{len(test_subs)}\")\n",
    "        train_ids = set(id2sub_df[id2sub_df.Subject.isin(train_subs)].index)\n",
    "        test_ids = set(id2sub_df[id2sub_df.Subject.isin(test_subs)].index)\n",
    "        fold_train_fpaths[module].append([f for f in paths if basename(f).split('.')[0] in train_ids])\n",
    "        fold_valid_fpaths[module].append([f for f in paths if basename(f).split('.')[0] in test_ids])\n",
    "        del train_subs, test_subs, train_ids, test_ids\n",
    "        gc.collect()\n",
    "    del sub_train_df\n",
    "del train_df\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6b7de674",
   "metadata": {
    "papermill": {
     "duration": 0.007341,
     "end_time": "2023-04-08T14:40:45.767183",
     "exception": false,
     "start_time": "2023-04-08T14:40:45.759842",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "79cf37e1",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-04-08T14:40:45.783927Z",
     "iopub.status.busy": "2023-04-08T14:40:45.783619Z",
     "iopub.status.idle": "2023-04-08T14:40:45.803895Z",
     "shell.execute_reply": "2023-04-08T14:40:45.802947Z"
    },
    "papermill": {
     "duration": 0.030641,
     "end_time": "2023-04-08T14:40:45.805933",
     "exception": false,
     "start_time": "2023-04-08T14:40:45.775292",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Adapted from FOGDataset of https://www.kaggle.com/code/mayukh18/pytorch-fog-end-to-end-baseline-lb-0-254\n",
    "class FOGSequence(tf.keras.utils.Sequence):\n",
    "\n",
    "    def __init__(self, df_paths, cfg=cfg, split=\"train\"):\n",
    "        _time = perf_counter()\n",
    "        \n",
    "        self.rng = default_rng(42)\n",
    "        self.cfg = cfg\n",
    "        self.split = split\n",
    "        \n",
    "        self.past_pad = self.cfg.wx*(self.cfg.window_past-1)\n",
    "        self.future_pad = self.cfg.wx*self.cfg.window_future\n",
    "        \n",
    "        if self.split == \"test\":\n",
    "            self.Ids = []\n",
    "        _values = [self._read(f) for f in df_paths]\n",
    "        \n",
    "        self.mapping = []\n",
    "        _length = 0\n",
    "        for _value in _values:\n",
    "            _shape = _value.shape[0]\n",
    "            self.mapping.extend(range(_length+self.past_pad, _length+_shape-self.future_pad))\n",
    "            _length += _shape\n",
    "            \n",
    "        self.values = np.concatenate(_values, axis=0)\n",
    "        self.mapping = np.array(self.mapping)\n",
    "        if self.split != \"test\":\n",
    "            # Keep only vaild and task rows\n",
    "            _valid_pos = self.values[self.mapping,self.valid_position] > 0\n",
    "            _task_pos = self.values[self.mapping,self.task_position] > 0\n",
    "            self.mapping = self.mapping[_valid_pos&_task_pos]\n",
    "        self.length = self.mapping.shape[0]\n",
    "        \n",
    "        print(f\"Valid Dataset of size {self.length:,} initialized in {perf_counter() - _time:.3f} secs!\")\n",
    "        gc.collect()\n",
    "    \n",
    "    def _read(self, path):\n",
    "        _is_tdcs = basename(dirname(path)).startswith('tdcs')\n",
    "        df = pd.read_csv(path)\n",
    "        \n",
    "        if self.split == \"test\":\n",
    "            _ids = basename(path).split('.')[0] + '_' + df.Time.astype(str)\n",
    "            self.Ids.extend(_ids.tolist())\n",
    "            return self._df_to_array(df, self.cfg.feature_list)\n",
    "        \n",
    "        _cols = [*self.cfg.feature_list, *self.cfg.label_list, 'Valid', 'Task']\n",
    "        self.valid_position = self.cfg.n_features + self.cfg.n_labels\n",
    "        self.task_position = self.valid_position + 1\n",
    "        \n",
    "        if _is_tdcs:\n",
    "            # Fill Valid and Task columns for tdcsfog\n",
    "            df['Valid'] = 1\n",
    "            df['Task'] = 1\n",
    "            \n",
    "        return self._df_to_array(df, _cols)\n",
    "    \n",
    "    def _df_to_array(self, df, cols):\n",
    "        # Pads past and future rows to dataframe values for indexing \n",
    "        _values = df[cols].values.astype(np.float16)\n",
    "        return np.pad(_values, ((self.past_pad, self.future_pad),(0,0)), 'edge')\n",
    "    \n",
    "    def __len__(self):\n",
    "        return int(np.ceil(self.length / self.cfg.batch_size))\n",
    "    \n",
    "    def __getitem__(self, idx):\n",
    "        \n",
    "        if self.split == \"train\":\n",
    "            # Onlt train set has randomly selected batches\n",
    "            _idxs = self.rng.choice(self.mapping, size=self.cfg.batch_size, replace=False)\n",
    "        else:\n",
    "            _idxs = self._get_indices(idx)\n",
    "            \n",
    "        # For test return only features\n",
    "        if self.split == \"test\":\n",
    "            return self._get_X(_idxs)\n",
    "        # For train and val splits return y also\n",
    "        return self._get_X_y(_idxs)\n",
    "    \n",
    "    def _get_indices(self, idx):\n",
    "        _low = idx * self.cfg.batch_size\n",
    "        # Cap high at self.length so overflow does not occur\n",
    "        _high = min(_low + self.cfg.batch_size, self.length)\n",
    "        return self.mapping[_low:_high]\n",
    "    \n",
    "    def _get_X(self, indices):\n",
    "        _X = np.empty((len(indices), self.cfg.window_size, self.cfg.n_features), dtype=np.float16)\n",
    "        for i, idx in enumerate(indices):\n",
    "            _X[i] = self.values[idx-self.past_pad:idx+self.future_pad+1:self.cfg.wx, :self.cfg.n_features]\n",
    "        return _X\n",
    "    \n",
    "    def _get_X_y(self, indices):\n",
    "        _X = np.empty((len(indices), self.cfg.window_size, self.cfg.n_features), dtype=np.float16)\n",
    "        for i, idx in enumerate(indices):\n",
    "            _X[i] = self.values[idx-self.past_pad: idx+self.future_pad+1:self.cfg.wx, :self.cfg.n_features]\n",
    "        return _X, self.values[indices, self.cfg.n_features:self.cfg.n_features+self.cfg.n_labels]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4532f866",
   "metadata": {
    "papermill": {
     "duration": 0.00698,
     "end_time": "2023-04-08T14:40:45.820194",
     "exception": false,
     "start_time": "2023-04-08T14:40:45.813214",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "f61cea7c",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-04-08T14:40:45.836617Z",
     "iopub.status.busy": "2023-04-08T14:40:45.835735Z",
     "iopub.status.idle": "2023-04-08T14:40:45.841672Z",
     "shell.execute_reply": "2023-04-08T14:40:45.840795Z"
    },
    "papermill": {
     "duration": 0.016336,
     "end_time": "2023-04-08T14:40:45.843757",
     "exception": false,
     "start_time": "2023-04-08T14:40:45.827421",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# average_precision_score with positive sample added if no true positive cases are present\n",
    "def calculate_precision(y_true, y_pred):\n",
    "    pad_width = ((0,0),(0,0)) if y_true.any(axis=0).all() else ((1,0),(0,0))\n",
    "    y_true, y_pred = np.pad(y_true, pad_width, constant_values=1), np.pad(y_pred, pad_width, constant_values=1)\n",
    "    return average_precision_score(y_true, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "d8f916f2",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-04-08T14:40:45.859819Z",
     "iopub.status.busy": "2023-04-08T14:40:45.858958Z",
     "iopub.status.idle": "2023-04-08T14:40:45.866594Z",
     "shell.execute_reply": "2023-04-08T14:40:45.865694Z"
    },
    "papermill": {
     "duration": 0.017773,
     "end_time": "2023-04-08T14:40:45.868626",
     "exception": false,
     "start_time": "2023-04-08T14:40:45.850853",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Note: Not same result as average_precision_score\n",
    "class AveragePrecision(tf.keras.metrics.Metric):\n",
    "\n",
    "    def __init__(self, num_classes, thresholds=None, name='avg_precision', **kwargs):\n",
    "        super(AveragePrecision, self).__init__(name=name, **kwargs)\n",
    "        self.class_precision = [tf.keras.metrics.Precision(thresholds) for _ in range(num_classes)]\n",
    "\n",
    "    def update_state(self, y_true, y_pred, sample_weight=None):\n",
    "        for i, precision in enumerate(self.class_precision):\n",
    "            precision.update_state(y_true[...,i], y_pred[..., i])\n",
    "\n",
    "    def result(self):\n",
    "        return tf.math.reduce_mean([precision.result() for precision in self.class_precision])\n",
    "    \n",
    "    def reset_state(self):\n",
    "        for precision in self.class_precision:\n",
    "            precision.reset_state()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "66a95775",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-04-08T14:40:45.884738Z",
     "iopub.status.busy": "2023-04-08T14:40:45.883827Z",
     "iopub.status.idle": "2023-04-08T14:40:48.550419Z",
     "shell.execute_reply": "2023-04-08T14:40:48.549575Z"
    },
    "papermill": {
     "duration": 2.697329,
     "end_time": "2023-04-08T14:40:48.573132",
     "exception": false,
     "start_time": "2023-04-08T14:40:45.875803",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential\"\n",
      "_________________________________________________________________\n",
      " Layer (type)                Output Shape              Param #   \n",
      "=================================================================\n",
      " conv1d (Conv1D)             (None, 64, 128)           5888      \n",
      "                                                                 \n",
      " batch_normalization (BatchN  (None, 64, 128)          512       \n",
      " ormalization)                                                   \n",
      "                                                                 \n",
      " re_lu (ReLU)                (None, 64, 128)           0         \n",
      "                                                                 \n",
      " dropout (Dropout)           (None, 64, 128)           0         \n",
      "                                                                 \n",
      " conv1d_1 (Conv1D)           (None, 64, 128)           245888    \n",
      "                                                                 \n",
      " batch_normalization_1 (Batc  (None, 64, 128)          512       \n",
      " hNormalization)                                                 \n",
      "                                                                 \n",
      " re_lu_1 (ReLU)              (None, 64, 128)           0         \n",
      "                                                                 \n",
      " dropout_1 (Dropout)         (None, 64, 128)           0         \n",
      "                                                                 \n",
      " conv1d_2 (Conv1D)           (None, 64, 128)           245888    \n",
      "                                                                 \n",
      " batch_normalization_2 (Batc  (None, 64, 128)          512       \n",
      " hNormalization)                                                 \n",
      "                                                                 \n",
      " re_lu_2 (ReLU)              (None, 64, 128)           0         \n",
      "                                                                 \n",
      " dropout_2 (Dropout)         (None, 64, 128)           0         \n",
      "                                                                 \n",
      " global_average_pooling1d (G  (None, 128)              0         \n",
      " lobalAveragePooling1D)                                          \n",
      "                                                                 \n",
      " dense (Dense)               (None, 3)                 387       \n",
      "                                                                 \n",
      "=================================================================\n",
      "Total params: 499,587\n",
      "Trainable params: 498,819\n",
      "Non-trainable params: 768\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# Model adapted from https://keras.io/examples/timeseries/timeseries_classification_from_scratch/\n",
    "def get_model(checkpoint_path = None):\n",
    "    model = tf.keras.models.Sequential()\n",
    "    model.add(tf.keras.Input(shape=(cfg.window_size, cfg.n_features), dtype='float16'))\n",
    "    for _ in range(cfg.model_nblocks):\n",
    "        model.add(tf.keras.layers.Conv1D(filters=cfg.model_hidden, kernel_size=15, padding=\"same\"))\n",
    "        model.add(tf.keras.layers.BatchNormalization())\n",
    "        model.add(tf.keras.layers.ReLU())\n",
    "        model.add(tf.keras.layers.Dropout(cfg.model_dropout))\n",
    "    model.add(tf.keras.layers.GlobalAveragePooling1D())\n",
    "    model.add(tf.keras.layers.Dense(cfg.n_labels, activation=None))\n",
    "\n",
    "    if checkpoint_path is not None:\n",
    "        model.load_weights(checkpoint_path)\n",
    "    model.compile(\n",
    "        tf.keras.optimizers.Adam(learning_rate=cfg.lr), \n",
    "        loss = tf.keras.losses.BinaryCrossentropy(from_logits=True),\n",
    "        metrics=[AveragePrecision(cfg.n_labels, thresholds=0.0)]\n",
    "    )\n",
    "    return model\n",
    "\n",
    "get_model().summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8466a3ec",
   "metadata": {
    "papermill": {
     "duration": 0.009052,
     "end_time": "2023-04-08T14:40:48.591639",
     "exception": false,
     "start_time": "2023-04-08T14:40:48.582587",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "00f562b2",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-04-08T14:40:48.611181Z",
     "iopub.status.busy": "2023-04-08T14:40:48.610893Z",
     "iopub.status.idle": "2023-04-08T14:40:48.618519Z",
     "shell.execute_reply": "2023-04-08T14:40:48.617605Z"
    },
    "papermill": {
     "duration": 0.019764,
     "end_time": "2023-04-08T14:40:48.620567",
     "exception": false,
     "start_time": "2023-04-08T14:40:48.600803",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def predict_select_model(fold, ds, model_save_dir=''):\n",
    "    best_path, best_score = None, -1\n",
    "    print(f\"Validation for fold{fold}:\")\n",
    "    for model_path in sorted(glob(join(model_save_dir, f\"fold{fold}_*.h5\"))):\n",
    "        pred_time = perf_counter()\n",
    "        gc.collect()\n",
    "        score = calculate_precision(\n",
    "            ds.values[ds.mapping, cfg.n_features:cfg.n_features + cfg.n_labels], \n",
    "            expit(get_model(model_path).predict(ds, verbose=0)) # expit converts to sigmoid output\n",
    "        )\n",
    "        if best_score < score:\n",
    "            best_score = score\n",
    "            best_path = model_path\n",
    "        gc.collect()\n",
    "        print(\"\\t\", basename(model_path), f\": score-{score:.4f} in {(perf_counter()-pred_time)/60:.2f} mins\")\n",
    "    print(basename(best_path), \"selected with score\", best_score)\n",
    "    return best_path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "62f3a0b2",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-04-08T14:40:48.640858Z",
     "iopub.status.busy": "2023-04-08T14:40:48.640040Z",
     "iopub.status.idle": "2023-04-08T14:40:48.646993Z",
     "shell.execute_reply": "2023-04-08T14:40:48.646082Z"
    },
    "papermill": {
     "duration": 0.019384,
     "end_time": "2023-04-08T14:40:48.649044",
     "exception": false,
     "start_time": "2023-04-08T14:40:48.629660",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def train_loop(train_paths, valid_paths, fold, model_save_dir=''):\n",
    "    gc.collect()\n",
    "    \n",
    "    # train_paths, test_paths = train_test_split(train_paths, test_size=0.4)\n",
    "    train_ds = FOGSequence(train_paths)\n",
    "    val_ds = FOGSequence(valid_paths, split=\"val\")\n",
    "    # test_ds = FOGSequence(test_paths, split=\"val\")\n",
    "    \n",
    "    model = get_model()\n",
    "    ckpt = tf.keras.callbacks.ModelCheckpoint(join(model_save_dir, f\"fold{fold}_model_\"+\"{epoch:02d}.h5\"), save_weights_only=True)\n",
    "    history = model.fit(train_ds, epochs=cfg.num_epochs, verbose=2, workers=5, validation_data=val_ds, use_multiprocessing=True, callbacks=[ckpt])\n",
    "    \n",
    "    best_model_path = predict_select_model(fold, val_ds, model_save_dir)\n",
    "    # score = calculate_precision(test_ds.values[test_ds.mapping, cfg.n_features:cfg.n_features + cfg.n_labels], expit(get_model(best_model_path).predict(test_ds, verbose=0)))\n",
    "    \n",
    "    del train_ds, val_ds, model, ckpt, history\n",
    "    gc.collect()\n",
    "    return best_model_path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "df6d463d",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-04-08T14:40:48.668580Z",
     "iopub.status.busy": "2023-04-08T14:40:48.668313Z",
     "iopub.status.idle": "2023-04-08T21:56:57.546141Z",
     "shell.execute_reply": "2023-04-08T21:56:57.545078Z"
    },
    "papermill": {
     "duration": 26168.890595,
     "end_time": "2023-04-08T21:56:57.548944",
     "exception": false,
     "start_time": "2023-04-08T14:40:48.658349",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "***Training defog***************************************************************************\n",
      "Fold 0-------------------------\n",
      "Valid Dataset of size 3,613,642 initialized in 15.419 secs!\n",
      "Valid Dataset of size 497,680 initialized in 2.032 secs!\n",
      "Epoch 1/5\n",
      "3529/3529 - 186s - loss: 0.0934 - avg_precision: 0.2306 - val_loss: 0.3015 - val_avg_precision: 0.0169 - 186s/epoch - 53ms/step\n",
      "Epoch 2/5\n",
      "3529/3529 - 176s - loss: 0.0650 - avg_precision: 0.4587 - val_loss: 0.2571 - val_avg_precision: 0.0194 - 176s/epoch - 50ms/step\n",
      "Epoch 3/5\n",
      "3529/3529 - 176s - loss: 0.0571 - avg_precision: 0.4949 - val_loss: 0.3718 - val_avg_precision: 0.3525 - 176s/epoch - 50ms/step\n",
      "Epoch 4/5\n",
      "3529/3529 - 176s - loss: 0.0509 - avg_precision: 0.5267 - val_loss: 0.3762 - val_avg_precision: 0.1265 - 176s/epoch - 50ms/step\n",
      "Epoch 5/5\n",
      "3529/3529 - 176s - loss: 0.0472 - avg_precision: 0.5372 - val_loss: 0.3312 - val_avg_precision: 0.0988 - 176s/epoch - 50ms/step\n",
      "Validation for fold0:\n",
      "\t fold0_model_01.h5 : score-0.3830 in 0.12 mins\n",
      "\t fold0_model_02.h5 : score-0.3999 in 0.11 mins\n",
      "\t fold0_model_03.h5 : score-0.4183 in 0.12 mins\n",
      "\t fold0_model_04.h5 : score-0.4031 in 0.11 mins\n",
      "\t fold0_model_05.h5 : score-0.4183 in 0.11 mins\n",
      "fold0_model_03.h5 selected with score 0.4183338855718253\n",
      "Fold 0 done in 17.00 min\n",
      "Fold 1-------------------------\n",
      "Valid Dataset of size 3,777,741 initialized in 16.743 secs!\n",
      "Valid Dataset of size 333,581 initialized in 1.565 secs!\n",
      "Epoch 1/5\n",
      "3690/3690 - 184s - loss: 0.1053 - avg_precision: 0.3439 - val_loss: 0.0874 - val_avg_precision: 0.0854 - 184s/epoch - 50ms/step\n",
      "Epoch 2/5\n",
      "3690/3690 - 181s - loss: 0.0709 - avg_precision: 0.4879 - val_loss: 0.1017 - val_avg_precision: 0.0278 - 181s/epoch - 49ms/step\n",
      "Epoch 3/5\n",
      "3690/3690 - 181s - loss: 0.0599 - avg_precision: 0.5199 - val_loss: 0.1071 - val_avg_precision: 0.0957 - 181s/epoch - 49ms/step\n",
      "Epoch 4/5\n",
      "3690/3690 - 180s - loss: 0.0530 - avg_precision: 0.5391 - val_loss: 0.1226 - val_avg_precision: 0.0786 - 180s/epoch - 49ms/step\n",
      "Epoch 5/5\n",
      "3690/3690 - 180s - loss: 0.0481 - avg_precision: 0.5510 - val_loss: 0.1578 - val_avg_precision: 0.0588 - 180s/epoch - 49ms/step\n",
      "Validation for fold1:\n",
      "\t fold1_model_01.h5 : score-0.4204 in 0.08 mins\n",
      "\t fold1_model_02.h5 : score-0.3859 in 0.08 mins\n",
      "\t fold1_model_03.h5 : score-0.4025 in 0.10 mins\n",
      "\t fold1_model_04.h5 : score-0.3918 in 0.08 mins\n",
      "\t fold1_model_05.h5 : score-0.3903 in 0.08 mins\n",
      "fold1_model_01.h5 selected with score 0.42039610681717327\n",
      "Fold 1 done in 16.56 min\n",
      "Fold 2-------------------------\n",
      "Valid Dataset of size 3,887,999 initialized in 16.207 secs!\n",
      "Valid Dataset of size 223,323 initialized in 1.097 secs!\n",
      "Epoch 1/5\n",
      "3797/3797 - 188s - loss: 0.0881 - avg_precision: 0.4616 - val_loss: 0.2408 - val_avg_precision: 0.1723 - 188s/epoch - 49ms/step\n",
      "Epoch 2/5\n",
      "3797/3797 - 184s - loss: 0.0621 - avg_precision: 0.5034 - val_loss: 0.2976 - val_avg_precision: 0.1427 - 184s/epoch - 48ms/step\n",
      "Epoch 3/5\n",
      "3797/3797 - 184s - loss: 0.0536 - avg_precision: 0.5265 - val_loss: 0.3140 - val_avg_precision: 0.1433 - 184s/epoch - 49ms/step\n",
      "Epoch 4/5\n",
      "3797/3797 - 184s - loss: 0.0476 - avg_precision: 0.5432 - val_loss: 0.3607 - val_avg_precision: 0.1517 - 184s/epoch - 49ms/step\n",
      "Epoch 5/5\n",
      "3797/3797 - 184s - loss: 0.0431 - avg_precision: 0.5552 - val_loss: 0.5289 - val_avg_precision: 0.1224 - 184s/epoch - 49ms/step\n",
      "Validation for fold2:\n",
      "\t fold2_model_01.h5 : score-0.5096 in 0.06 mins\n",
      "\t fold2_model_02.h5 : score-0.5173 in 0.06 mins\n",
      "\t fold2_model_03.h5 : score-0.5189 in 0.05 mins\n",
      "\t fold2_model_04.h5 : score-0.5241 in 0.05 mins\n",
      "\t fold2_model_05.h5 : score-0.5228 in 0.06 mins\n",
      "fold2_model_04.h5 selected with score 0.5241413735898507\n",
      "Fold 2 done in 16.56 min\n",
      "Fold 3-------------------------\n",
      "Valid Dataset of size 3,775,121 initialized in 16.373 secs!\n",
      "Valid Dataset of size 336,201 initialized in 1.502 secs!\n",
      "Epoch 1/5\n",
      "3687/3687 - 184s - loss: 0.1001 - avg_precision: 0.3964 - val_loss: 0.2507 - val_avg_precision: 0.3128 - 184s/epoch - 50ms/step\n",
      "Epoch 2/5\n",
      "3687/3687 - 184s - loss: 0.0688 - avg_precision: 0.4743 - val_loss: 0.3213 - val_avg_precision: 0.3313 - 184s/epoch - 50ms/step\n",
      "Epoch 3/5\n",
      "3687/3687 - 184s - loss: 0.0573 - avg_precision: 0.5123 - val_loss: 0.3856 - val_avg_precision: 0.3159 - 184s/epoch - 50ms/step\n",
      "Epoch 4/5\n",
      "3687/3687 - 184s - loss: 0.0497 - avg_precision: 0.5331 - val_loss: 0.2929 - val_avg_precision: 0.1900 - 184s/epoch - 50ms/step\n",
      "Epoch 5/5\n",
      "3687/3687 - 184s - loss: 0.0452 - avg_precision: 0.5471 - val_loss: 0.3341 - val_avg_precision: 0.2845 - 184s/epoch - 50ms/step\n",
      "Validation for fold3:\n",
      "\t fold3_model_01.h5 : score-0.1450 in 0.08 mins\n",
      "\t fold3_model_02.h5 : score-0.1463 in 0.08 mins\n",
      "\t fold3_model_03.h5 : score-0.1395 in 0.09 mins\n",
      "\t fold3_model_04.h5 : score-0.1193 in 0.08 mins\n",
      "\t fold3_model_05.h5 : score-0.1259 in 0.08 mins\n",
      "fold3_model_02.h5 selected with score 0.14630013332536154\n",
      "Fold 3 done in 17.29 min\n",
      "Fold 4-------------------------\n",
      "Valid Dataset of size 3,835,130 initialized in 16.910 secs!\n",
      "Valid Dataset of size 276,192 initialized in 1.112 secs!\n",
      "Epoch 1/5\n",
      "3746/3746 - 189s - loss: 0.1143 - avg_precision: 0.2291 - val_loss: 0.1301 - val_avg_precision: 0.0000e+00 - 189s/epoch - 51ms/step\n",
      "Epoch 2/5\n",
      "3746/3746 - 186s - loss: 0.0680 - avg_precision: 0.4913 - val_loss: 0.1965 - val_avg_precision: 0.0000e+00 - 186s/epoch - 50ms/step\n",
      "Epoch 3/5\n",
      "3746/3746 - 187s - loss: 0.0540 - avg_precision: 0.5302 - val_loss: 0.2323 - val_avg_precision: 0.0000e+00 - 187s/epoch - 50ms/step\n",
      "Epoch 4/5\n",
      "3746/3746 - 186s - loss: 0.0480 - avg_precision: 0.5463 - val_loss: 0.2182 - val_avg_precision: 0.0000e+00 - 186s/epoch - 50ms/step\n",
      "Epoch 5/5\n",
      "3746/3746 - 186s - loss: 0.0434 - avg_precision: 0.5592 - val_loss: 0.2952 - val_avg_precision: 2.6062e-04 - 186s/epoch - 50ms/step\n",
      "Validation for fold4:\n",
      "\t fold4_model_01.h5 : score-0.3712 in 0.07 mins\n",
      "\t fold4_model_02.h5 : score-0.3674 in 0.07 mins\n",
      "\t fold4_model_03.h5 : score-0.3648 in 0.07 mins\n",
      "\t fold4_model_04.h5 : score-0.3668 in 0.07 mins\n",
      "\t fold4_model_05.h5 : score-0.3637 in 0.07 mins\n",
      "fold4_model_01.h5 selected with score 0.3711943743009318\n",
      "Fold 4 done in 16.75 min\n",
      "Fold 5-------------------------\n",
      "Valid Dataset of size 3,571,312 initialized in 15.528 secs!\n",
      "Valid Dataset of size 540,010 initialized in 2.198 secs!\n",
      "Epoch 1/5\n",
      "3488/3488 - 180s - loss: 0.0987 - avg_precision: 0.3371 - val_loss: 0.1589 - val_avg_precision: 0.0692 - 180s/epoch - 52ms/step\n",
      "Epoch 2/5\n",
      "3488/3488 - 177s - loss: 0.0612 - avg_precision: 0.5071 - val_loss: 0.1687 - val_avg_precision: 0.0943 - 177s/epoch - 51ms/step\n",
      "Epoch 3/5\n",
      "3488/3488 - 176s - loss: 0.0523 - avg_precision: 0.5343 - val_loss: 0.2118 - val_avg_precision: 0.0325 - 176s/epoch - 51ms/step\n",
      "Epoch 4/5\n",
      "3488/3488 - 176s - loss: 0.0450 - avg_precision: 0.5535 - val_loss: 0.2676 - val_avg_precision: 0.0504 - 176s/epoch - 50ms/step\n",
      "Epoch 5/5\n",
      "3488/3488 - 174s - loss: 0.0410 - avg_precision: 0.5643 - val_loss: 0.4065 - val_avg_precision: 0.0265 - 174s/epoch - 50ms/step\n",
      "Validation for fold5:\n",
      "\t fold5_model_01.h5 : score-0.4424 in 0.12 mins\n",
      "\t fold5_model_02.h5 : score-0.4423 in 0.12 mins\n",
      "\t fold5_model_03.h5 : score-0.4365 in 0.12 mins\n",
      "\t fold5_model_04.h5 : score-0.4223 in 0.12 mins\n",
      "\t fold5_model_05.h5 : score-0.3985 in 0.12 mins\n",
      "fold5_model_01.h5 selected with score 0.4423991318372749\n",
      "Fold 5 done in 16.45 min\n",
      "Fold 6-------------------------\n",
      "Valid Dataset of size 3,651,971 initialized in 15.763 secs!\n",
      "Valid Dataset of size 459,351 initialized in 2.326 secs!\n",
      "Epoch 1/5\n",
      "3567/3567 - 180s - loss: 0.1143 - avg_precision: 0.3918 - val_loss: 0.1421 - val_avg_precision: 0.0303 - 180s/epoch - 50ms/step\n",
      "Epoch 2/5\n",
      "3567/3567 - 176s - loss: 0.0762 - avg_precision: 0.4718 - val_loss: 0.0995 - val_avg_precision: 3.3748e-05 - 176s/epoch - 49ms/step\n",
      "Epoch 3/5\n",
      "3567/3567 - 176s - loss: 0.0632 - avg_precision: 0.5131 - val_loss: 0.1059 - val_avg_precision: 0.0086 - 176s/epoch - 49ms/step\n",
      "Epoch 4/5\n",
      "3567/3567 - 178s - loss: 0.0527 - avg_precision: 0.5415 - val_loss: 0.1208 - val_avg_precision: 0.0054 - 178s/epoch - 50ms/step\n",
      "Epoch 5/5\n",
      "3567/3567 - 176s - loss: 0.0481 - avg_precision: 0.5522 - val_loss: 0.1460 - val_avg_precision: 0.0180 - 176s/epoch - 49ms/step\n",
      "Validation for fold6:\n",
      "\t fold6_model_01.h5 : score-0.3641 in 0.11 mins\n",
      "\t fold6_model_02.h5 : score-0.3629 in 0.11 mins\n",
      "\t fold6_model_03.h5 : score-0.3677 in 0.10 mins\n",
      "\t fold6_model_04.h5 : score-0.3599 in 0.10 mins\n",
      "\t fold6_model_05.h5 : score-0.3611 in 0.10 mins\n",
      "fold6_model_03.h5 selected with score 0.3677353611323931\n",
      "Fold 6 done in 16.87 min\n",
      "Fold 7-------------------------\n",
      "Valid Dataset of size 3,875,135 initialized in 16.753 secs!\n",
      "Valid Dataset of size 236,187 initialized in 1.261 secs!\n",
      "Epoch 1/5\n",
      "3785/3785 - 187s - loss: 0.1206 - avg_precision: 0.2361 - val_loss: 0.2415 - val_avg_precision: 0.0432 - 187s/epoch - 49ms/step\n",
      "Epoch 2/5\n",
      "3785/3785 - 184s - loss: 0.0702 - avg_precision: 0.4820 - val_loss: 0.2200 - val_avg_precision: 0.0590 - 184s/epoch - 49ms/step\n",
      "Epoch 3/5\n",
      "3785/3785 - 184s - loss: 0.0576 - avg_precision: 0.5193 - val_loss: 0.1315 - val_avg_precision: 0.0162 - 184s/epoch - 49ms/step\n",
      "Epoch 4/5\n",
      "3785/3785 - 184s - loss: 0.0483 - avg_precision: 0.5474 - val_loss: 0.1324 - val_avg_precision: 0.0508 - 184s/epoch - 49ms/step\n",
      "Epoch 5/5\n",
      "3785/3785 - 184s - loss: 0.0431 - avg_precision: 0.7612 - val_loss: 0.1347 - val_avg_precision: 0.0310 - 184s/epoch - 49ms/step\n",
      "Validation for fold7:\n",
      "\t fold7_model_01.h5 : score-0.3720 in 0.06 mins\n",
      "\t fold7_model_02.h5 : score-0.3789 in 0.06 mins\n",
      "\t fold7_model_03.h5 : score-0.3745 in 0.06 mins\n",
      "\t fold7_model_04.h5 : score-0.3859 in 0.06 mins\n",
      "\t fold7_model_05.h5 : score-0.3830 in 0.06 mins\n",
      "fold7_model_04.h5 selected with score 0.38586291240366394\n",
      "Fold 7 done in 16.61 min\n",
      "Fold 8-------------------------\n",
      "Valid Dataset of size 3,421,802 initialized in 14.808 secs!\n",
      "Valid Dataset of size 689,520 initialized in 2.912 secs!\n",
      "Epoch 1/5\n",
      "3342/3342 - 173s - loss: 0.1049 - avg_precision: 0.3810 - val_loss: 0.0891 - val_avg_precision: 0.2165 - 173s/epoch - 52ms/step\n",
      "Epoch 2/5\n",
      "3342/3342 - 170s - loss: 0.0673 - avg_precision: 0.5025 - val_loss: 0.1345 - val_avg_precision: 0.1194 - 170s/epoch - 51ms/step\n",
      "Epoch 3/5\n",
      "3342/3342 - 171s - loss: 0.0568 - avg_precision: 0.5295 - val_loss: 0.1527 - val_avg_precision: 0.0703 - 171s/epoch - 51ms/step\n",
      "Epoch 4/5\n",
      "3342/3342 - 173s - loss: 0.0478 - avg_precision: 0.5540 - val_loss: 0.1167 - val_avg_precision: 0.1546 - 173s/epoch - 52ms/step\n",
      "Epoch 5/5\n",
      "3342/3342 - 172s - loss: 0.0456 - avg_precision: 0.8904 - val_loss: 0.2144 - val_avg_precision: 0.1090 - 172s/epoch - 51ms/step\n",
      "Validation for fold8:\n",
      "\t fold8_model_01.h5 : score-0.4935 in 0.16 mins\n",
      "\t fold8_model_02.h5 : score-0.4292 in 0.15 mins\n",
      "\t fold8_model_03.h5 : score-0.4283 in 0.15 mins\n",
      "\t fold8_model_04.h5 : score-0.4596 in 0.15 mins\n",
      "\t fold8_model_05.h5 : score-0.4092 in 0.15 mins\n",
      "fold8_model_01.h5 selected with score 0.49345038378866146\n",
      "Fold 8 done in 16.38 min\n",
      "Fold 9-------------------------\n",
      "Valid Dataset of size 3,592,045 initialized in 15.376 secs!\n",
      "Valid Dataset of size 519,277 initialized in 2.080 secs!\n",
      "Epoch 1/5\n",
      "3508/3508 - 178s - loss: 0.1057 - avg_precision: 0.4280 - val_loss: 0.1510 - val_avg_precision: 0.0633 - 178s/epoch - 51ms/step\n",
      "Epoch 2/5\n",
      "3508/3508 - 174s - loss: 0.0663 - avg_precision: 0.4995 - val_loss: 0.1542 - val_avg_precision: 0.1286 - 174s/epoch - 50ms/step\n",
      "Epoch 3/5\n",
      "3508/3508 - 176s - loss: 0.0587 - avg_precision: 0.5209 - val_loss: 0.1078 - val_avg_precision: 0.1546 - 176s/epoch - 50ms/step\n",
      "Epoch 4/5\n",
      "3508/3508 - 174s - loss: 0.0487 - avg_precision: 0.5506 - val_loss: 0.1645 - val_avg_precision: 0.1136 - 174s/epoch - 50ms/step\n",
      "Epoch 5/5\n",
      "3508/3508 - 174s - loss: 0.0447 - avg_precision: 0.5596 - val_loss: 0.1725 - val_avg_precision: 0.1378 - 174s/epoch - 50ms/step\n",
      "Validation for fold9:\n",
      "\t fold9_model_01.h5 : score-0.4015 in 0.12 mins\n",
      "\t fold9_model_02.h5 : score-0.4368 in 0.11 mins\n",
      "\t fold9_model_03.h5 : score-0.4549 in 0.12 mins\n",
      "\t fold9_model_04.h5 : score-0.4134 in 0.11 mins\n",
      "\t fold9_model_05.h5 : score-0.4442 in 0.11 mins\n",
      "fold9_model_03.h5 selected with score 0.4549283832876039\n",
      "Fold 9 done in 17.32 min\n",
      "***defog done in 2.80 hrs**************************************************\n",
      "\n",
      "***Training tdcsfog***************************************************************************\n",
      "Fold 0-------------------------\n",
      "Valid Dataset of size 6,244,003 initialized in 7.325 secs!\n",
      "Valid Dataset of size 818,669 initialized in 0.904 secs!\n",
      "Epoch 1/5\n",
      "6098/6098 - 300s - loss: 0.1345 - avg_precision: 0.7500 - val_loss: 0.2417 - val_avg_precision: 0.5459 - 300s/epoch - 49ms/step\n",
      "Epoch 2/5\n",
      "6098/6098 - 297s - loss: 0.0867 - avg_precision: 0.8528 - val_loss: 0.2573 - val_avg_precision: 0.3060 - 297s/epoch - 49ms/step\n",
      "Epoch 3/5\n",
      "6098/6098 - 297s - loss: 0.0645 - avg_precision: 0.8957 - val_loss: 0.2616 - val_avg_precision: 0.2991 - 297s/epoch - 49ms/step\n",
      "Epoch 4/5\n",
      "6098/6098 - 298s - loss: 0.0522 - avg_precision: 0.9188 - val_loss: 0.2959 - val_avg_precision: 0.3074 - 298s/epoch - 49ms/step\n",
      "Epoch 5/5\n",
      "6098/6098 - 300s - loss: 0.0483 - avg_precision: 0.9244 - val_loss: 0.3660 - val_avg_precision: 0.2823 - 300s/epoch - 49ms/step\n",
      "Validation for fold0:\n",
      "\t fold0_model_01.h5 : score-0.2857 in 0.20 mins\n",
      "\t fold0_model_02.h5 : score-0.2984 in 0.18 mins\n",
      "\t fold0_model_03.h5 : score-0.2981 in 0.17 mins\n",
      "\t fold0_model_04.h5 : score-0.3052 in 0.18 mins\n",
      "\t fold0_model_05.h5 : score-0.2746 in 0.17 mins\n",
      "fold0_model_04.h5 selected with score 0.3051864972421526\n",
      "Fold 0 done in 26.74 min\n",
      "Fold 1-------------------------\n",
      "Valid Dataset of size 5,456,205 initialized in 6.687 secs!\n",
      "Valid Dataset of size 1,606,467 initialized in 1.562 secs!\n",
      "Epoch 1/5\n",
      "5329/5329 - 277s - loss: 0.0933 - avg_precision: 0.7124 - val_loss: 0.9471 - val_avg_precision: 0.2092 - 277s/epoch - 52ms/step\n",
      "Epoch 2/5\n",
      "5329/5329 - 290s - loss: 0.0631 - avg_precision: 0.8096 - val_loss: 1.2769 - val_avg_precision: 0.2222 - 290s/epoch - 54ms/step\n",
      "Epoch 3/5\n",
      "5329/5329 - 274s - loss: 0.0513 - avg_precision: 0.8436 - val_loss: 1.1722 - val_avg_precision: 0.2405 - 274s/epoch - 51ms/step\n",
      "Epoch 4/5\n",
      "5329/5329 - 274s - loss: 0.0441 - avg_precision: 0.8667 - val_loss: 1.4301 - val_avg_precision: 0.2639 - 274s/epoch - 51ms/step\n",
      "Epoch 5/5\n",
      "5329/5329 - 275s - loss: 0.0368 - avg_precision: 0.8883 - val_loss: 1.6812 - val_avg_precision: 0.2385 - 275s/epoch - 52ms/step\n",
      "Validation for fold1:\n",
      "\t fold1_model_01.h5 : score-0.3012 in 0.35 mins\n",
      "\t fold1_model_02.h5 : score-0.3022 in 0.38 mins\n",
      "\t fold1_model_03.h5 : score-0.3058 in 0.33 mins\n",
      "\t fold1_model_04.h5 : score-0.3031 in 0.34 mins\n",
      "\t fold1_model_05.h5 : score-0.3151 in 0.34 mins\n",
      "fold1_model_05.h5 selected with score 0.3151388625838063\n",
      "Fold 1 done in 25.59 min\n",
      "Fold 2-------------------------\n",
      "Valid Dataset of size 6,371,516 initialized in 7.937 secs!\n",
      "Valid Dataset of size 691,156 initialized in 0.912 secs!\n",
      "Epoch 1/5\n",
      "6223/6223 - 307s - loss: 0.1349 - avg_precision: 0.7883 - val_loss: 0.1565 - val_avg_precision: 0.1502 - 307s/epoch - 49ms/step\n",
      "Epoch 2/5\n",
      "6223/6223 - 306s - loss: 0.0910 - avg_precision: 0.8566 - val_loss: 0.2116 - val_avg_precision: 0.1540 - 306s/epoch - 49ms/step\n",
      "Epoch 3/5\n",
      "6223/6223 - 305s - loss: 0.0729 - avg_precision: 0.8900 - val_loss: 0.1578 - val_avg_precision: 0.1885 - 305s/epoch - 49ms/step\n",
      "Epoch 4/5\n",
      "6223/6223 - 305s - loss: 0.0615 - avg_precision: 0.9096 - val_loss: 0.1374 - val_avg_precision: 0.2551 - 305s/epoch - 49ms/step\n",
      "Epoch 5/5\n",
      "6223/6223 - 304s - loss: 0.0537 - avg_precision: 0.9217 - val_loss: 0.1724 - val_avg_precision: 0.2471 - 304s/epoch - 49ms/step\n",
      "Validation for fold2:\n",
      "\t fold2_model_01.h5 : score-0.1309 in 0.16 mins\n",
      "\t fold2_model_02.h5 : score-0.1395 in 0.15 mins\n",
      "\t fold2_model_03.h5 : score-0.2059 in 0.15 mins\n",
      "\t fold2_model_04.h5 : score-0.2454 in 0.15 mins\n",
      "\t fold2_model_05.h5 : score-0.2351 in 0.15 mins\n",
      "fold2_model_04.h5 selected with score 0.24538202694291025\n",
      "Fold 2 done in 27.22 min\n",
      "Fold 3-------------------------\n",
      "Valid Dataset of size 6,561,037 initialized in 8.517 secs!\n",
      "Valid Dataset of size 501,635 initialized in 0.665 secs!\n",
      "Epoch 1/5\n",
      "6408/6408 - 313s - loss: 0.1342 - avg_precision: 0.7676 - val_loss: 0.1653 - val_avg_precision: 0.2715 - 313s/epoch - 49ms/step\n",
      "Epoch 2/5\n",
      "6408/6408 - 308s - loss: 0.0872 - avg_precision: 0.8563 - val_loss: 0.1771 - val_avg_precision: 0.5791 - 308s/epoch - 48ms/step\n",
      "Epoch 3/5\n",
      "6408/6408 - 309s - loss: 0.0703 - avg_precision: 0.8892 - val_loss: 0.2020 - val_avg_precision: 0.2921 - 309s/epoch - 48ms/step\n",
      "Epoch 4/5\n",
      "6408/6408 - 309s - loss: 0.0582 - avg_precision: 0.9118 - val_loss: 0.2421 - val_avg_precision: 0.1954 - 309s/epoch - 48ms/step\n",
      "Epoch 5/5\n",
      "6408/6408 - 310s - loss: 0.0509 - avg_precision: 0.9236 - val_loss: 0.2572 - val_avg_precision: 0.1586 - 310s/epoch - 48ms/step\n",
      "Validation for fold3:\n",
      "\t fold3_model_01.h5 : score-0.2379 in 0.12 mins\n",
      "\t fold3_model_02.h5 : score-0.3083 in 0.11 mins\n",
      "\t fold3_model_03.h5 : score-0.2145 in 0.12 mins\n",
      "\t fold3_model_04.h5 : score-0.2096 in 0.11 mins\n",
      "\t fold3_model_05.h5 : score-0.1639 in 0.11 mins\n",
      "fold3_model_02.h5 selected with score 0.3082504827906002\n",
      "Fold 3 done in 26.76 min\n",
      "Fold 4-------------------------\n",
      "Valid Dataset of size 6,711,663 initialized in 7.926 secs!\n",
      "Valid Dataset of size 351,009 initialized in 0.525 secs!\n",
      "Epoch 1/5\n",
      "6555/6555 - 318s - loss: 0.1382 - avg_precision: 0.7725 - val_loss: 0.0977 - val_avg_precision: 0.1367 - 318s/epoch - 49ms/step\n",
      "Epoch 2/5\n",
      "6555/6555 - 314s - loss: 0.0879 - avg_precision: 0.8590 - val_loss: 0.0522 - val_avg_precision: 0.2843 - 314s/epoch - 48ms/step\n",
      "Epoch 3/5\n",
      "6555/6555 - 314s - loss: 0.0712 - avg_precision: 0.8897 - val_loss: 0.0830 - val_avg_precision: 0.2565 - 314s/epoch - 48ms/step\n",
      "Epoch 4/5\n",
      "6555/6555 - 315s - loss: 0.0596 - avg_precision: 0.9098 - val_loss: 0.1008 - val_avg_precision: 0.2067 - 315s/epoch - 48ms/step\n",
      "Epoch 5/5\n",
      "6555/6555 - 313s - loss: 0.0498 - avg_precision: 0.9262 - val_loss: 0.0765 - val_avg_precision: 0.3215 - 313s/epoch - 48ms/step\n",
      "Validation for fold4:\n",
      "\t fold4_model_01.h5 : score-0.2606 in 0.09 mins\n",
      "\t fold4_model_02.h5 : score-0.2801 in 0.08 mins\n",
      "\t fold4_model_03.h5 : score-0.2627 in 0.08 mins\n",
      "\t fold4_model_04.h5 : score-0.2559 in 0.08 mins\n",
      "\t fold4_model_05.h5 : score-0.2751 in 0.08 mins\n",
      "fold4_model_02.h5 selected with score 0.28010785437705354\n",
      "Fold 4 done in 27.08 min\n",
      "Fold 5-------------------------\n",
      "Valid Dataset of size 6,403,889 initialized in 8.200 secs!\n",
      "Valid Dataset of size 658,783 initialized in 0.894 secs!\n",
      "Epoch 1/5\n",
      "6254/6254 - 307s - loss: 0.1324 - avg_precision: 0.7707 - val_loss: 0.2960 - val_avg_precision: 0.2702 - 307s/epoch - 49ms/step\n",
      "Epoch 2/5\n",
      "6254/6254 - 303s - loss: 0.0850 - avg_precision: 0.8580 - val_loss: 0.3917 - val_avg_precision: 0.5239 - 303s/epoch - 49ms/step\n",
      "Epoch 3/5\n",
      "6254/6254 - 304s - loss: 0.0703 - avg_precision: 0.8850 - val_loss: 0.3636 - val_avg_precision: 0.4273 - 304s/epoch - 49ms/step\n",
      "Epoch 4/5\n",
      "6254/6254 - 303s - loss: 0.0571 - avg_precision: 0.9114 - val_loss: 0.4047 - val_avg_precision: 0.3309 - 303s/epoch - 48ms/step\n",
      "Epoch 5/5\n",
      "6254/6254 - 303s - loss: 0.0495 - avg_precision: 0.9243 - val_loss: 0.4204 - val_avg_precision: 0.3433 - 303s/epoch - 48ms/step\n",
      "Validation for fold5:\n",
      "\t fold5_model_01.h5 : score-0.1738 in 0.15 mins\n",
      "\t fold5_model_02.h5 : score-0.2039 in 0.14 mins\n",
      "\t fold5_model_03.h5 : score-0.2275 in 0.14 mins\n",
      "\t fold5_model_04.h5 : score-0.2079 in 0.14 mins\n",
      "\t fold5_model_05.h5 : score-0.1959 in 0.14 mins\n",
      "fold5_model_03.h5 selected with score 0.22753604912831057\n",
      "Fold 5 done in 26.24 min\n",
      "Fold 6-------------------------\n",
      "Valid Dataset of size 6,310,688 initialized in 7.499 secs!\n",
      "Valid Dataset of size 751,984 initialized in 0.848 secs!\n",
      "Epoch 1/5\n",
      "6163/6163 - 304s - loss: 0.1332 - avg_precision: 0.7411 - val_loss: 0.2143 - val_avg_precision: 0.2079 - 304s/epoch - 49ms/step\n",
      "Epoch 2/5\n",
      "6163/6163 - 300s - loss: 0.0827 - avg_precision: 0.8643 - val_loss: 0.2494 - val_avg_precision: 0.1810 - 300s/epoch - 49ms/step\n",
      "Epoch 3/5\n",
      "6163/6163 - 300s - loss: 0.0716 - avg_precision: 0.8839 - val_loss: 0.2695 - val_avg_precision: 0.1772 - 300s/epoch - 49ms/step\n",
      "Epoch 4/5\n",
      "6163/6163 - 299s - loss: 0.0566 - avg_precision: 0.9126 - val_loss: 0.2535 - val_avg_precision: 0.1827 - 299s/epoch - 49ms/step\n",
      "Epoch 5/5\n",
      "6163/6163 - 300s - loss: 0.0479 - avg_precision: 0.9271 - val_loss: 0.2862 - val_avg_precision: 0.1850 - 300s/epoch - 49ms/step\n",
      "Validation for fold6:\n",
      "\t fold6_model_01.h5 : score-0.1714 in 0.17 mins\n",
      "\t fold6_model_02.h5 : score-0.1772 in 0.16 mins\n",
      "\t fold6_model_03.h5 : score-0.1735 in 0.17 mins\n",
      "\t fold6_model_04.h5 : score-0.1675 in 0.16 mins\n",
      "\t fold6_model_05.h5 : score-0.1890 in 0.16 mins\n",
      "fold6_model_05.h5 selected with score 0.1889508470751585\n",
      "Fold 6 done in 26.39 min\n",
      "Fold 7-------------------------\n",
      "Valid Dataset of size 6,500,745 initialized in 8.100 secs!\n",
      "Valid Dataset of size 561,927 initialized in 0.744 secs!\n",
      "Epoch 1/5\n",
      "6349/6349 - 310s - loss: 0.1421 - avg_precision: 0.7370 - val_loss: 0.1637 - val_avg_precision: 0.2615 - 310s/epoch - 49ms/step\n",
      "Epoch 2/5\n",
      "6349/6349 - 307s - loss: 0.0944 - avg_precision: 0.8435 - val_loss: 0.1487 - val_avg_precision: 0.2728 - 307s/epoch - 48ms/step\n",
      "Epoch 3/5\n",
      "6349/6349 - 307s - loss: 0.0737 - avg_precision: 0.8843 - val_loss: 0.1827 - val_avg_precision: 0.2919 - 307s/epoch - 48ms/step\n",
      "Epoch 4/5\n",
      "6349/6349 - 306s - loss: 0.0631 - avg_precision: 0.9024 - val_loss: 0.1728 - val_avg_precision: 0.2910 - 306s/epoch - 48ms/step\n",
      "Epoch 5/5\n",
      "6349/6349 - 307s - loss: 0.0542 - avg_precision: 0.9182 - val_loss: 0.1925 - val_avg_precision: 0.2853 - 307s/epoch - 48ms/step\n",
      "Validation for fold7:\n",
      "\t fold7_model_01.h5 : score-0.2535 in 0.13 mins\n",
      "\t fold7_model_02.h5 : score-0.2585 in 0.12 mins\n",
      "\t fold7_model_03.h5 : score-0.2631 in 0.13 mins\n",
      "\t fold7_model_04.h5 : score-0.2604 in 0.12 mins\n",
      "\t fold7_model_05.h5 : score-0.2614 in 0.12 mins\n",
      "fold7_model_03.h5 selected with score 0.2631028019874581\n",
      "Fold 7 done in 27.16 min\n",
      "Fold 8-------------------------\n",
      "Valid Dataset of size 6,484,199 initialized in 7.652 secs!\n",
      "Valid Dataset of size 578,473 initialized in 0.921 secs!\n",
      "Epoch 1/5\n",
      "6333/6333 - 310s - loss: 0.1476 - avg_precision: 0.7115 - val_loss: 0.3600 - val_avg_precision: 0.0753 - 310s/epoch - 49ms/step\n",
      "Epoch 2/5\n",
      "6333/6333 - 306s - loss: 0.0960 - avg_precision: 0.8441 - val_loss: 0.2438 - val_avg_precision: 0.1010 - 306s/epoch - 48ms/step\n",
      "Epoch 3/5\n",
      "6333/6333 - 307s - loss: 0.0751 - avg_precision: 0.8847 - val_loss: 0.1213 - val_avg_precision: 0.1916 - 307s/epoch - 48ms/step\n",
      "Epoch 4/5\n",
      "6333/6333 - 307s - loss: 0.0623 - avg_precision: 0.9062 - val_loss: 0.1397 - val_avg_precision: 0.1702 - 307s/epoch - 48ms/step\n",
      "Epoch 5/5\n",
      "6333/6333 - 307s - loss: 0.0530 - avg_precision: 0.9213 - val_loss: 0.1280 - val_avg_precision: 0.2318 - 307s/epoch - 48ms/step\n",
      "Validation for fold8:\n",
      "\t fold8_model_01.h5 : score-0.0648 in 0.13 mins\n",
      "\t fold8_model_02.h5 : score-0.0979 in 0.13 mins\n",
      "\t fold8_model_03.h5 : score-0.1890 in 0.13 mins\n",
      "\t fold8_model_04.h5 : score-0.1685 in 0.13 mins\n",
      "\t fold8_model_05.h5 : score-0.1943 in 0.13 mins\n",
      "fold8_model_05.h5 selected with score 0.19428959309763727\n",
      "Fold 8 done in 27.68 min\n",
      "Fold 9-------------------------\n",
      "Valid Dataset of size 6,520,103 initialized in 7.806 secs!\n",
      "Valid Dataset of size 542,569 initialized in 0.831 secs!\n",
      "Epoch 1/5\n",
      "6368/6368 - 314s - loss: 0.1362 - avg_precision: 0.7803 - val_loss: 0.1176 - val_avg_precision: 0.1302 - 314s/epoch - 49ms/step\n",
      "Epoch 2/5\n",
      "6368/6368 - 312s - loss: 0.0877 - avg_precision: 0.8618 - val_loss: 0.1217 - val_avg_precision: 0.1174 - 312s/epoch - 49ms/step\n",
      "Epoch 3/5\n",
      "6368/6368 - 308s - loss: 0.0700 - avg_precision: 0.8927 - val_loss: 0.1432 - val_avg_precision: 0.1191 - 308s/epoch - 48ms/step\n",
      "Epoch 4/5\n",
      "6368/6368 - 308s - loss: 0.0609 - avg_precision: 0.9079 - val_loss: 0.1316 - val_avg_precision: 0.2750 - 308s/epoch - 48ms/step\n",
      "Epoch 5/5\n",
      "6368/6368 - 307s - loss: 0.0508 - avg_precision: 0.9249 - val_loss: 0.1442 - val_avg_precision: 0.1310 - 307s/epoch - 48ms/step\n",
      "Validation for fold9:\n",
      "\t fold9_model_01.h5 : score-0.1106 in 0.12 mins\n",
      "\t fold9_model_02.h5 : score-0.1573 in 0.12 mins\n",
      "\t fold9_model_03.h5 : score-0.1491 in 0.13 mins\n",
      "\t fold9_model_04.h5 : score-0.1483 in 0.12 mins\n",
      "\t fold9_model_05.h5 : score-0.1442 in 0.12 mins\n",
      "fold9_model_02.h5 selected with score 0.15727250574617688\n",
      "Fold 9 done in 27.48 min\n",
      "***tdcsfog done in 4.47 hrs**************************************************\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Main training loop\n",
    "model_paths = {'defog': [], 'tdcsfog':[]}\n",
    "for module in model_paths:\n",
    "    module_start = perf_counter()\n",
    "    print(f\"***Training {module}{'*'*75}\")\n",
    "    if not exists(module): \n",
    "        os.mkdir(module)\n",
    "    for fold, (train_fpaths, valid_fpaths) in enumerate(zip(fold_train_fpaths[module], fold_valid_fpaths[module])):\n",
    "        fold_start = perf_counter()\n",
    "        print(f\"Fold {fold}{'-'*25}\")\n",
    "        model_paths[module].append(train_loop(train_fpaths, valid_fpaths, fold, model_save_dir=module))\n",
    "        print(f\"Fold {fold} done in {(perf_counter()-fold_start)/60:.2f} min\")\n",
    "    print(f\"***{module} done in {(perf_counter()-module_start)/3600:.2f} hrs{'*'*50}\\n\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5c4ec267",
   "metadata": {
    "papermill": {
     "duration": 0.026813,
     "end_time": "2023-04-08T21:56:57.606027",
     "exception": false,
     "start_time": "2023-04-08T21:56:57.579214",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Submission"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "1be453e2",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-04-08T21:56:57.672298Z",
     "iopub.status.busy": "2023-04-08T21:56:57.671951Z",
     "iopub.status.idle": "2023-04-08T21:57:36.925452Z",
     "shell.execute_reply": "2023-04-08T21:57:36.924375Z"
    },
    "papermill": {
     "duration": 39.295395,
     "end_time": "2023-04-08T21:57:36.928336",
     "exception": false,
     "start_time": "2023-04-08T21:56:57.632941",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Valid Dataset of size 281,688 initialized in 0.540 secs!\n",
      "Valid Dataset of size 4,682 initialized in 0.020 secs!\n"
     ]
    }
   ],
   "source": [
    "test_defog_paths = glob(join(TEST_DIR, \"defog/*.csv\"))\n",
    "test_tdcsfog_paths = glob(join(TEST_DIR, \"tdcsfog/*.csv\"))\n",
    "\n",
    "test_ds_dict = {\n",
    "    'defog':FOGSequence(test_defog_paths, split=\"test\"), \n",
    "    'tdcsfog':FOGSequence(test_tdcsfog_paths, split=\"test\")\n",
    "}\n",
    "\n",
    "# Get test predictions\n",
    "df_list = []\n",
    "for module, test_ds in test_ds_dict.items():\n",
    "    y_pred_list = []\n",
    "    for model_path in model_paths[module]:\n",
    "        model = get_model(model_path)\n",
    "        y_pred_list.append(expit(model.predict(test_ds, verbose=0)))  # expit converts to sigmoid output\n",
    "    y_pred = np.mean(y_pred_list, axis=0)\n",
    "    df_list.append(pd.DataFrame(\n",
    "        {'Id': test_ds.Ids, 'StartHesitation': y_pred[:,0], 'Turn': y_pred[:,1], 'Walking': y_pred[:,2]}))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "143878e4",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-04-08T21:57:36.985335Z",
     "iopub.status.busy": "2023-04-08T21:57:36.985016Z",
     "iopub.status.idle": "2023-04-08T21:57:39.151162Z",
     "shell.execute_reply": "2023-04-08T21:57:39.150105Z"
    },
    "papermill": {
     "duration": 2.197175,
     "end_time": "2023-04-08T21:57:39.154030",
     "exception": false,
     "start_time": "2023-04-08T21:57:36.956855",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Concatenate Prediction to DataFrames\n",
    "submission = pd.concat(df_list)\n",
    "\n",
    "# Only keep Ids in sample_submission\n",
    "sample_submission = pd.read_csv(join(BASE_DIR, \"sample_submission.csv\"))\n",
    "submission = pd.merge(sample_submission[['Id']], submission, how='left', on='Id').fillna(0.0)\n",
    "submission.to_csv(\"submission.csv\", index=False, float_format='%.5f') # round to 5 decimal places while keeping point notation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "30d80278",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-04-08T21:57:39.210142Z",
     "iopub.status.busy": "2023-04-08T21:57:39.209818Z",
     "iopub.status.idle": "2023-04-08T21:57:40.314864Z",
     "shell.execute_reply": "2023-04-08T21:57:40.313624Z"
    },
    "papermill": {
     "duration": 1.135544,
     "end_time": "2023-04-08T21:57:40.317412",
     "exception": false,
     "start_time": "2023-04-08T21:57:39.181868",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Id,StartHesitation,Turn,Walking\r\n",
      "003f117e14_0,0.00019,0.00004,0.00006\r\n",
      "003f117e14_1,0.00019,0.00004,0.00006\r\n",
      "003f117e14_2,0.00019,0.00004,0.00006\r\n",
      "003f117e14_3,0.00019,0.00004,0.00006\r\n"
     ]
    }
   ],
   "source": [
    "!head -5 submission.csv"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.12"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 26345.280193,
   "end_time": "2023-04-08T21:57:43.742796",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2023-04-08T14:38:38.462603",
   "version": "2.4.0"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "state": {
     "1cacb7246aa14591893bcc7a08f9d2ee": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_a8e2b2a69f5e40129db912f0e3569d52",
       "placeholder": "​",
       "style": "IPY_MODEL_e1ea30dfb3194f80afb78ea97c8cd26f",
       "value": " 970/970 [00:34&lt;00:00, 71.09it/s]"
      }
     },
     "2bb68e4192c24d8dac58e420f90fd18b": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_a877186f8c9e4ec8a39c2c52f2136fd1",
        "IPY_MODEL_7c9c0681773b493685448c5fcd6533da",
        "IPY_MODEL_1cacb7246aa14591893bcc7a08f9d2ee"
       ],
       "layout": "IPY_MODEL_c4df2fbf38134c92b362fcec71a2de89"
      }
     },
     "48c1a432a1934754993ea19b07b21b30": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "7c9c0681773b493685448c5fcd6533da": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_48c1a432a1934754993ea19b07b21b30",
       "max": 970.0,
       "min": 0.0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_c5b16f66e1ad4d3ead767b569e220263",
       "value": 970.0
      }
     },
     "a12886d13ec34b41ba9d76ac17c5bdde": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "a3a044363735452fb8a8aa9e86347c39": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "a877186f8c9e4ec8a39c2c52f2136fd1": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_a3a044363735452fb8a8aa9e86347c39",
       "placeholder": "​",
       "style": "IPY_MODEL_a12886d13ec34b41ba9d76ac17c5bdde",
       "value": "100%"
      }
     },
     "a8e2b2a69f5e40129db912f0e3569d52": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "c4df2fbf38134c92b362fcec71a2de89": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "c5b16f66e1ad4d3ead767b569e220263": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": ""
      }
     },
     "e1ea30dfb3194f80afb78ea97c8cd26f": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     }
    },
    "version_major": 2,
    "version_minor": 0
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
